<html><!-- Created using the cpp_pretty_printer from the dlib C++ library.  See http://dlib.net for updates. --><head><title>dlib C++ Library - layers.h</title></head><body bgcolor='white'><pre>
<font color='#009900'>// Copyright (C) 2015  Davis E. King (davis@dlib.net)
</font><font color='#009900'>// License: Boost Software License   See LICENSE.txt for the full license.
</font><font color='#0000FF'>#ifndef</font> DLIB_DNn_LAYERS_H_
<font color='#0000FF'>#define</font> DLIB_DNn_LAYERS_H_

<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='layers_abstract.h.html'>layers_abstract.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../cuda/tensor.h.html'>../cuda/tensor.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='core.h.html'>core.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>iostream<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>string<font color='#5555FF'>&gt;</font>
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../rand.h.html'>../rand.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../string.h.html'>../string.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../cuda/tensor_tools.h.html'>../cuda/tensor_tools.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='../vectorstream.h.html'>../vectorstream.h</a>"
<font color='#0000FF'>#include</font> "<a style='text-decoration:none' href='utilities.h.html'>utilities.h</a>"
<font color='#0000FF'>#include</font> <font color='#5555FF'>&lt;</font>sstream<font color='#5555FF'>&gt;</font>


<font color='#0000FF'>namespace</font> dlib
<b>{</b>

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>struct</font> <b><a name='num_con_outputs'></a>num_con_outputs</b>
    <b>{</b>
        <b><a name='num_con_outputs'></a>num_con_outputs</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n<font face='Lucida Console'>)</font> : num_outputs<font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font> <b>{</b><b>}</b>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _num_filters,
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='con_'></a>con_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_num_filters <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of filters must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr<font face='Lucida Console'>)</font>, "<font color='#CC0000'>The padding must be smaller than the filter size.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc<font face='Lucida Console'>)</font>, "<font color='#CC0000'>The padding must be smaller than the filter size.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#979000'>0</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> _padding_y, "<font color='#CC0000'>If _nr==0 then the padding must be set to 0 as well.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font color='#979000'>0</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> _padding_x, "<font color='#CC0000'>If _nr==0 then the padding must be set to 0 as well.</font>"<font face='Lucida Console'>)</font>;

        <b><a name='con_'></a>con_</b><font face='Lucida Console'>(</font>
            num_con_outputs o
        <font face='Lucida Console'>)</font> : 
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            num_filters_<font face='Lucida Console'>(</font>o.num_outputs<font face='Lucida Console'>)</font>,
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>,
            use_bias<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num_filters_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b><a name='con_'></a>con_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : con_<font face='Lucida Console'>(</font>num_con_outputs<font face='Lucida Console'>(</font>_num_filters<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='num_filters'></a>num_filters</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> num_filters_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nr<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> filters.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                <font color='#0000FF'>return</font> _nr;
        <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nc<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> filters.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                <font color='#0000FF'>return</font> _nc;
        <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='set_num_filters'></a>set_num_filters</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> num<font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>num <font color='#5555FF'>!</font><font color='#5555FF'>=</font> num_filters_<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, 
                    "<font color='#CC0000'>You can't change the number of filters in con_ if the parameter tensor has already been allocated.</font>"<font face='Lucida Console'>)</font>;
                num_filters_ <font color='#5555FF'>=</font> num;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>bool</u></font> <b><a name='bias_is_disabled'></a>bias_is_disabled</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#5555FF'>!</font>use_bias; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='disable_bias'></a>disable_bias</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>false</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            use_bias <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> num_filters_<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> temp <font color='#5555FF'>=</font> params;
            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> num_filters_<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>copy</font><font face='Lucida Console'>(</font>temp.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, temp.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> num_filters_, params.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='enable_bias'></a>enable_bias</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>true</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            use_bias <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> temp <font color='#5555FF'>=</font> params;
            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>params.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> num_filters_<font face='Lucida Console'>)</font>;
            std::<font color='#BB00BB'>copy</font><font face='Lucida Console'>(</font>temp.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, temp.<font color='#BB00BB'>end</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, params.<font color='#BB00BB'>begin</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, num_filters_<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='con_'></a>con_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : 
            params<font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>,
            filters<font face='Lucida Console'>(</font>item.filters<font face='Lucida Console'>)</font>,
            biases<font face='Lucida Console'>(</font>item.biases<font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font>item.learning_rate_multiplier<font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font>item.weight_decay_multiplier<font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier<font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier<font face='Lucida Console'>)</font>,
            num_filters_<font face='Lucida Console'>(</font>item.num_filters_<font face='Lucida Console'>)</font>,
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>,
            use_bias<font face='Lucida Console'>(</font>item.use_bias<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;conv is non-copyable and basically stateless, so we have to write our
</font>            <font color='#009900'>// own copy to avoid trying to copy it and getting an error.
</font>        <b>}</b>

        con_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            <font color='#009900'>// this-&gt;conv is non-copyable and basically stateless, so we have to write our
</font>            <font color='#009900'>// own copy to avoid trying to copy it and getting an error.
</font>            params <font color='#5555FF'>=</font> item.params;
            filters <font color='#5555FF'>=</font> item.filters;
            biases <font color='#5555FF'>=</font> item.biases;
            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;
            learning_rate_multiplier <font color='#5555FF'>=</font> item.learning_rate_multiplier;
            weight_decay_multiplier <font color='#5555FF'>=</font> item.weight_decay_multiplier;
            bias_learning_rate_multiplier <font color='#5555FF'>=</font> item.bias_learning_rate_multiplier;
            bias_weight_decay_multiplier <font color='#5555FF'>=</font> item.bias_weight_decay_multiplier;
            num_filters_ <font color='#5555FF'>=</font> item.num_filters_;
            use_bias <font color='#5555FF'>=</font> item.use_bias;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> filt_nr <font color='#5555FF'>=</font> _nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> ? _nr : sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>long</u></font> filt_nc <font color='#5555FF'>=</font> _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> ? _nc : sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>long</u></font> num_inputs <font color='#5555FF'>=</font> filt_nr<font color='#5555FF'>*</font>filt_nc<font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> num_outputs <font color='#5555FF'>=</font> num_filters_;
            <font color='#009900'>// allocate params for the filters and also for the filter bias values.
</font>            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs<font color='#5555FF'>*</font>num_filters_ <font color='#5555FF'>+</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> num_filters_<font face='Lucida Console'>)</font>;

            dlib::rand <font color='#BB00BB'>rnd</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>rand</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>randomize_parameters</font><font face='Lucida Console'>(</font>params, num_inputs<font color='#5555FF'>+</font>num_outputs, rnd<font face='Lucida Console'>)</font>;

            filters <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>num_filters_, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filt_nr, filt_nc<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,num_filters_<font face='Lucida Console'>)</font>;
                <font color='#009900'>// set the initial bias values to zero
</font>                <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            conv.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                       <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>,
                       _stride_y,
                       _stride_x,
                       padding_y_,
                       padding_x_<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output,
                    sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                    <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>,
                    <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#BB00BB'>conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output,
                    sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                    <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            conv.<font color='#BB00BB'>get_gradient_for_data</font> <font face='Lucida Console'>(</font><font color='#979000'>true</font>, gradient_input, <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// no point computing the parameter gradients if they won't be used.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>learning_rate_multiplier <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> filt <font color='#5555FF'>=</font> <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params_grad,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                conv.<font color='#BB00BB'>get_gradient_for_filters</font> <font face='Lucida Console'>(</font><font color='#979000'>false</font>, gradient_input, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, filt<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params_grad, filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    tt::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>b, gradient_input<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>con_5</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_filters_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.filters, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.biases, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.use_bias, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>con_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>con_4</font>" <font color='#5555FF'>|</font><font color='#5555FF'>|</font> version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>con_5</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_filters_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.filters, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.biases, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>con_5</font>"<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.use_bias, in<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::con_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>con\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_filters=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_filters_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> use_bias=false</font>";
            <b>}</b>
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> con_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;con</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_filters='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_filters_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> use_bias='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>item.use_bias?"<font color='#CC0000'>true</font>":"<font color='#CC0000'>false</font>"<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/con&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        resizable_tensor params;
        alias_tensor filters, biases;

        tt::tensor_conv conv;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
        <font color='#0000FF'><u>long</u></font> num_filters_;

        <font color='#009900'>// These are here only because older versions of con (which you might encounter
</font>        <font color='#009900'>// serialized to disk) used different padding settings.
</font>        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;
        <font color='#0000FF'><u>bool</u></font> use_bias;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> num_filters,
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> con <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>con_<font color='#5555FF'>&lt;</font>num_filters,nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _num_filters,
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='cont_'></a>cont_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:

        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_num_filters <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of filters must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr, "<font color='#CC0000'>The padding must be smaller than the filter size.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc, "<font color='#CC0000'>The padding must be smaller than the filter size.</font>"<font face='Lucida Console'>)</font>;

        <b><a name='cont_'></a>cont_</b><font face='Lucida Console'>(</font>
            num_con_outputs o
        <font face='Lucida Console'>)</font> : 
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            num_filters_<font face='Lucida Console'>(</font>o.num_outputs<font face='Lucida Console'>)</font>,
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>,
            use_bias<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num_filters_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <b><a name='cont_'></a>cont_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : cont_<font face='Lucida Console'>(</font>num_con_outputs<font face='Lucida Console'>(</font>_num_filters<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='num_filters'></a>num_filters</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> num_filters_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nr; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nc; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='set_num_filters'></a>set_num_filters</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> num<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>num <font color='#5555FF'>!</font><font color='#5555FF'>=</font> num_filters_<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>,
                    "<font color='#CC0000'>You can't change the number of filters in cont_ if the parameter tensor has already been allocated.</font>"<font face='Lucida Console'>)</font>;
                num_filters_ <font color='#5555FF'>=</font> num;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='disable_bias'></a>disable_bias</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> use_bias <font color='#5555FF'>=</font> <font color='#979000'>false</font>; <b>}</b>
        <font color='#0000FF'><u>bool</u></font> <b><a name='bias_is_disabled'></a>bias_is_disabled</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#5555FF'>!</font>use_bias; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='cont_'></a>cont_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> cont_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : 
            params<font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>,
            filters<font face='Lucida Console'>(</font>item.filters<font face='Lucida Console'>)</font>,
            biases<font face='Lucida Console'>(</font>item.biases<font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font>item.learning_rate_multiplier<font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font>item.weight_decay_multiplier<font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier<font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier<font face='Lucida Console'>)</font>,
            num_filters_<font face='Lucida Console'>(</font>item.num_filters_<font face='Lucida Console'>)</font>,
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>,
            use_bias<font face='Lucida Console'>(</font>item.use_bias<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;conv is non-copyable and basically stateless, so we have to write our
</font>            <font color='#009900'>// own copy to avoid trying to copy it and getting an error.
</font>        <b>}</b>

        cont_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> cont_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            <font color='#009900'>// this-&gt;conv is non-copyable and basically stateless, so we have to write our
</font>            <font color='#009900'>// own copy to avoid trying to copy it and getting an error.
</font>            params <font color='#5555FF'>=</font> item.params;
            filters <font color='#5555FF'>=</font> item.filters;
            biases <font color='#5555FF'>=</font> item.biases;
            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;
            learning_rate_multiplier <font color='#5555FF'>=</font> item.learning_rate_multiplier;
            weight_decay_multiplier <font color='#5555FF'>=</font> item.weight_decay_multiplier;
            bias_learning_rate_multiplier <font color='#5555FF'>=</font> item.bias_learning_rate_multiplier;
            bias_weight_decay_multiplier <font color='#5555FF'>=</font> item.bias_weight_decay_multiplier;
            num_filters_ <font color='#5555FF'>=</font> item.num_filters_;
            use_bias <font color='#5555FF'>=</font> item.use_bias;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'><u>long</u></font> num_inputs <font color='#5555FF'>=</font> _nr<font color='#5555FF'>*</font>_nc<font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> num_outputs <font color='#5555FF'>=</font> num_filters_;
            <font color='#009900'>// allocate params for the filters and also for the filter bias values.
</font>            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs<font color='#5555FF'>*</font>num_filters_ <font color='#5555FF'>+</font> num_filters_ <font color='#5555FF'>*</font> <font color='#0000FF'>static_cast</font><font color='#5555FF'>&lt;</font><font color='#0000FF'><u>int</u></font><font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            dlib::rand <font color='#BB00BB'>rnd</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>rand</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>randomize_parameters</font><font face='Lucida Console'>(</font>params, num_inputs<font color='#5555FF'>+</font>num_outputs, rnd<font face='Lucida Console'>)</font>;

            filters <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, num_filters_, _nr, _nc<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,num_filters_<font face='Lucida Console'>)</font>;
                <font color='#009900'>// set the initial bias values to zero
</font>                <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> filt <font color='#5555FF'>=</font> <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> gnr <font color='#5555FF'>=</font> _stride_y <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> filt.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font> <font color='#5555FF'>*</font> padding_y_;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> gnc <font color='#5555FF'>=</font> _stride_x <font color='#5555FF'>*</font> <font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> filt.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#979000'>2</font> <font color='#5555FF'>*</font> padding_x_;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> gnsamps <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>int</u></font> gk <font color='#5555FF'>=</font> filt.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>gnsamps,gk,gnr,gnc<font face='Lucida Console'>)</font>;
            conv.<font color='#BB00BB'>setup</font><font face='Lucida Console'>(</font>output,filt,_stride_y,_stride_x,padding_y_,padding_x_<font face='Lucida Console'>)</font>;
            conv.<font color='#BB00BB'>get_gradient_for_data</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,filt,output<font face='Lucida Console'>)</font>;            
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,output,<font color='#979000'>1</font>,<font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> filt <font color='#5555FF'>=</font> <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;           
            <font color='#BB00BB'>conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,gradient_input, filt<font face='Lucida Console'>)</font>;
            <font color='#009900'>// no point computing the parameter gradients if they won't be used.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>learning_rate_multiplier <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> filt <font color='#5555FF'>=</font> <font color='#BB00BB'>filters</font><font face='Lucida Console'>(</font>params_grad,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;                
                conv.<font color='#BB00BB'>get_gradient_for_filters</font> <font face='Lucida Console'>(</font><font color='#979000'>false</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,gradient_input, filt<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>use_bias<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params_grad, filters.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    tt::<font color='#BB00BB'>assign_conv_bias_gradient</font><font face='Lucida Console'>(</font>b, gradient_input<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> cont_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>cont_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_filters_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.filters, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.biases, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.use_bias, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>cont_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>cont_1</font>" <font color='#5555FF'>|</font><font color='#5555FF'>|</font> version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>cont_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_filters_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.filters, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.biases, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::con_</font>"<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>cont_2</font>"<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.use_bias, in<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::con_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>


        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> cont_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>cont\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_filters=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_filters_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> use_bias=false</font>";
            <b>}</b>
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> cont_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;cont</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_filters='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_filters_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> use_bias='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>item.use_bias?"<font color='#CC0000'>true</font>":"<font color='#CC0000'>false</font>"<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/cont&gt;</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        resizable_tensor params;
        alias_tensor filters, biases;

        tt::tensor_conv conv;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
        <font color='#0000FF'><u>long</u></font> num_filters_;

        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;

        <font color='#0000FF'><u>bool</u></font> use_bias;

    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> num_filters,
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> cont <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>cont_<font color='#5555FF'>&lt;</font>num_filters,nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>int</u></font> scale_y, 
        <font color='#0000FF'><u>int</u></font> scale_x 
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='upsample_'></a>upsample_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>scale_y <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>upsampling scale factor can't be less than 1.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>scale_x <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>upsampling scale factor can't be less than 1.</font>"<font face='Lucida Console'>)</font>;

        <b><a name='upsample_'></a>upsample_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                scale_y<font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                scale_x<font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>scale_x;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>scale_y;
            <font color='#0000FF'>return</font> p; 
        <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale_x;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale_y;
            <font color='#0000FF'>return</font> p; 
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> upsample_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>upsample_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>scale_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>scale_x, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>upsample_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>upsample_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::upsample_.</font>"<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>int</u></font> _scale_y;
            <font color='#0000FF'><u>int</u></font> _scale_x;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>_scale_y, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>_scale_x, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_scale_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> scale_y <font color='#5555FF'>|</font><font color='#5555FF'>|</font> _scale_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> scale_x<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong scale found while deserializing dlib::upsample_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> upsample_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>upsample\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>scale_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>scale_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, scale_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>scale_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> upsample_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;upsample</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> scale_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>scale_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> scale_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>scale_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>int</u></font> scale,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> upsample <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>upsample_<font color='#5555FF'>&lt;</font>scale,scale<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> NR_, 
        <font color='#0000FF'><u>long</u></font> NC_
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='resize_to_'></a>resize_to_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>NR_ <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>NR resize parameter can't be less than 1.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>NC_ <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>NC resize parameter can't be less than 1.</font>"<font face='Lucida Console'>)</font>;
        
        <b><a name='resize_to_'></a>resize_to_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>
        
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>
    
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            scale_y <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>NR_<font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            scale_x <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>NC_<font color='#5555FF'>/</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font><font face='Lucida Console'>)</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                NR_,
                NC_<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 
        
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>
        
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>scale_x;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>scale_y;
            <font color='#0000FF'>return</font> p; 
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> 
        <b>{</b> 
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale_x;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>scale_y;
            <font color='#0000FF'>return</font> p; 
        <b>}</b>
        
        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        
        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> resize_to_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>resize_to_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>NR_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>NC_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.scale_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.scale_x, out<font face='Lucida Console'>)</font>;
        <b>}</b>
        
        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>resize_to_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>resize_to_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::resize_to_.</font>"<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>long</u></font> _nr;
            <font color='#0000FF'><u>long</u></font> _nc;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>_nr, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>_nc, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.scale_y, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.scale_x, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> NR_ <font color='#5555FF'>|</font><font color='#5555FF'>|</font> _nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> NC_<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong size found while deserializing dlib::resize_to_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>
        
        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> resize_to_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>resize_to (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nr=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> NR_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> NC_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>
        
        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> resize_to_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;resize_to</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> NR_ <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'</font>" ;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> NC_ <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>double</u></font> scale_y;
        <font color='#0000FF'><u>double</u></font> scale_x;
    
    <b>}</b>;  <font color='#009900'>// end of class resize_to_
</font>    
    
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> NR,
        <font color='#0000FF'><u>long</u></font> NC,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> resize_to <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>resize_to_<font color='#5555FF'>&lt;</font>NR,NC<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    
<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='max_pool_'></a>max_pool_</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nc<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>public</font>:


        <b><a name='max_pool_'></a>max_pool_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nr; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nc; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='max_pool_'></a>max_pool_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>  :
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;mp is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>        <b>}</b>

        max_pool_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;

            <font color='#009900'>// this-&gt;mp is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            mp.<font color='#BB00BB'>setup_max_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>mp</font><font face='Lucida Console'>(</font>output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            mp.<font color='#BB00BB'>setup_max_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            mp.<font color='#BB00BB'>get_gradient</font><font face='Lucida Console'>(</font>gradient_input, computed_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>max_pool_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>max_pool_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>max_pool_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::max_pool_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::max_pool_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>max_pool (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> max_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;max_pool</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:


        tt::pooling mp;
        resizable_tensor params;

        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> max_pool <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>max_pool_<font color='#5555FF'>&lt;</font>nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> max_pool_everything <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>max_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#979000'>0</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc,
        <font color='#0000FF'><u>int</u></font> _stride_y,
        <font color='#0000FF'><u>int</u></font> _stride_x,
        <font color='#0000FF'><u>int</u></font> _padding_y <font color='#5555FF'>=</font> _stride_y<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nr<font color='#5555FF'>/</font><font color='#979000'>2</font>,
        <font color='#0000FF'><u>int</u></font> _padding_x <font color='#5555FF'>=</font> _stride_x<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>1</font>? <font color='#979000'>0</font> : _nc<font color='#5555FF'>/</font><font color='#979000'>2</font>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='avg_pool_'></a>avg_pool_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns in a filter must be &gt;= 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The filter stride must be &gt; 0</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_y <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_y <font color='#5555FF'>&lt;</font> _nr<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> _padding_x <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> <font face='Lucida Console'>(</font><font face='Lucida Console'>(</font>_nc<font color='#5555FF'>=</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>|</font><font color='#5555FF'>|</font> <font face='Lucida Console'>(</font>_nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> _padding_x <font color='#5555FF'>&lt;</font> _nc<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>, 
            "<font color='#CC0000'>The padding must be smaller than the filter size, unless the filters size is 0.</font>"<font face='Lucida Console'>)</font>;

        <b><a name='avg_pool_'></a>avg_pool_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> :
            padding_y_<font face='Lucida Console'>(</font>_padding_y<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>_padding_x<font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <font color='#0000FF'><u>long</u></font> <b><a name='nr'></a>nr</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nr; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='nc'></a>nc</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _nc; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_y'></a>stride_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_y; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='stride_x'></a>stride_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> _stride_x; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_y'></a>padding_y</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_y_; <b>}</b>
        <font color='#0000FF'><u>long</u></font> <b><a name='padding_x'></a>padding_x</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> padding_x_; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font><font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>-</font><font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>
            dpoint p
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font color='#BB00BB'>stride_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>padding_y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> <font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font><font color='#979000'>2</font>;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <b><a name='avg_pool_'></a>avg_pool_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>  :
            padding_y_<font face='Lucida Console'>(</font>item.padding_y_<font face='Lucida Console'>)</font>,
            padding_x_<font face='Lucida Console'>(</font>item.padding_x_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// this-&gt;ap is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>        <b>}</b>

        avg_pool_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            padding_y_ <font color='#5555FF'>=</font> item.padding_y_;
            padding_x_ <font color='#5555FF'>=</font> item.padding_x_;

            <font color='#009900'>// this-&gt;ap is non-copyable so we have to write our own copy to avoid trying to
</font>            <font color='#009900'>// copy it and getting an error.
</font>            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            ap.<font color='#BB00BB'>setup_avg_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>ap</font><font face='Lucida Console'>(</font>output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            ap.<font color='#BB00BB'>setup_avg_pooling</font><font face='Lucida Console'>(</font>_nr<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nr:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                                 _nc<font color='#5555FF'>!</font><font color='#5555FF'>=</font><font color='#979000'>0</font>?_nc:sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                 _stride_y, _stride_x, padding_y_, padding_x_<font face='Lucida Console'>)</font>;

            ap.<font color='#BB00BB'>get_gradient</font><font face='Lucida Console'>(</font>gradient_input, computed_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>avg_pool_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_y, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_stride_x, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_y_, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.padding_x_, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>avg_pool_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#0000FF'><u>int</u></font> stride_y;
            <font color='#0000FF'><u>int</u></font> stride_x;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>avg_pool_2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_y, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>stride_x, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_y_, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.padding_x_, in<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::avg_pool_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_y_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_y found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.padding_x_ <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _padding_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong padding_x found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_y <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_y<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_y found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>_stride_x <font color='#5555FF'>!</font><font color='#5555FF'>=</font> stride_x<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong stride_x found while deserializing dlib::avg_pool_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>avg_pool (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, stride_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_y=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, padding_x=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> avg_pool_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;avg_pool</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_y<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> stride_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_stride_x<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_y='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_y_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> padding_x='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.padding_x_<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:

        tt::pooling ap;
        resizable_tensor params;

        <font color='#0000FF'><u>int</u></font> padding_y_;
        <font color='#0000FF'><u>int</u></font> padding_x_;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'><u>int</u></font> stride_y,
        <font color='#0000FF'><u>int</u></font> stride_x,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> avg_pool <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>avg_pool_<font color='#5555FF'>&lt;</font>nr,nc,stride_y,stride_x<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> avg_pool_everything <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>avg_pool_<font color='#5555FF'>&lt;</font><font color='#979000'>0</font>,<font color='#979000'>0</font>,<font color='#979000'>1</font>,<font color='#979000'>1</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> DEFAULT_LAYER_NORM_EPS <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>;

    <font color='#0000FF'>class</font> <b><a name='layer_norm_'></a>layer_norm_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='layer_norm_'></a>layer_norm_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>double</u></font> eps_ <font color='#5555FF'>=</font> DEFAULT_LAYER_NORM_EPS
        <font face='Lucida Console'>)</font> :
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            eps<font face='Lucida Console'>(</font>eps_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_eps'></a>get_eps</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> eps; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            beta <font color='#5555FF'>=</font> gamma;

            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>beta.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>layer_normalize</font><font face='Lucida Console'>(</font>eps, output, means, invstds, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> g_grad <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params_grad, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b_grad <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params_grad, gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>layer_normalize_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means, invstds, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g_grad, b_grad<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>;
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>;

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> layer_norm_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>layer_norm_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gamma, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.beta, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.means, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.invstds, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.eps, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>layer_norm_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>layer_norm_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::layer_norm_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gamma, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.beta, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.means, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.invstds, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.eps, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> layer_norm_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer_norm</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> layer_norm_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>layer_norm</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/layer_norm&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        alias_tensor gamma, beta;
        resizable_tensor means, invstds;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> eps;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> layer_norm <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>layer_norm_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>    <font color='#0000FF'>enum</font> <b><a name='layer_mode'></a>layer_mode</b>
    <b>{</b>
        CONV_MODE <font color='#5555FF'>=</font> <font color='#979000'>0</font>,
        FC_MODE <font color='#5555FF'>=</font> <font color='#979000'>1</font>
    <b>}</b>;

    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> DEFAULT_BATCH_NORM_EPS <font color='#5555FF'>=</font> <font color='#979000'>0.0001</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        layer_mode mode
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='bn_'></a>bn_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='bn_'></a>bn_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> window_size,
            <font color='#0000FF'><u>double</u></font> eps_ <font color='#5555FF'>=</font> DEFAULT_BATCH_NORM_EPS
        <font face='Lucida Console'>)</font> : 
            num_updates<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>, 
            running_stats_window_size<font face='Lucida Console'>(</font>window_size<font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            eps<font face='Lucida Console'>(</font>eps_<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>window_size <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The batch normalization running stats window size can't be 0.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <b><a name='bn_'></a>bn_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : bn_<font face='Lucida Console'>(</font><font color='#979000'>100</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        layer_mode <b><a name='get_mode'></a>get_mode</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> mode; <b>}</b>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='get_running_stats_window_size'></a>get_running_stats_window_size</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> running_stats_window_size; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_running_stats_window_size'></a>set_running_stats_window_size</b> <font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> new_window_size <font face='Lucida Console'>)</font> 
        <b>{</b> 
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>new_window_size <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The batch normalization running stats window size can't be 0.</font>"<font face='Lucida Console'>)</font>;
            running_stats_window_size <font color='#5555FF'>=</font> new_window_size; 
        <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_eps'></a>get_eps</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> eps; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>


        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            beta <font color='#5555FF'>=</font> gamma;

            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>beta.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;

            running_means.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            running_variances.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font><font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            running_means <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            running_variances <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            num_updates <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> decay <font color='#5555FF'>=</font> <font color='#979000'>1.0</font> <font color='#5555FF'>-</font> num_updates<font color='#5555FF'>/</font><font face='Lucida Console'>(</font>num_updates<font color='#5555FF'>+</font><font color='#979000'>1.0</font><font face='Lucida Console'>)</font>;
                <font color='#5555FF'>+</font><font color='#5555FF'>+</font>num_updates;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>num_updates <font color='#5555FF'>&gt;</font> running_stats_window_size<font face='Lucida Console'>)</font>
                    num_updates <font color='#5555FF'>=</font> running_stats_window_size;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>batch_normalize</font><font face='Lucida Console'>(</font>eps, output, means, invstds, decay, running_means, running_variances, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font> 
                    tt::<font color='#BB00BB'>batch_normalize_conv</font><font face='Lucida Console'>(</font>eps, output, means, invstds, decay, running_means, running_variances, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#009900'>// we are running in testing mode so we just linearly scale the input tensor.
</font>            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>batch_normalize_inference</font><font face='Lucida Console'>(</font>eps, output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b, running_means, running_variances<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    tt::<font color='#BB00BB'>batch_normalize_conv_inference</font><font face='Lucida Console'>(</font>eps, output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, b, running_means, running_variances<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> g_grad <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params_grad, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b_grad <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params_grad, gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>batch_normalize_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means, invstds, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g_grad, b_grad <font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>batch_normalize_conv_gradient</font><font face='Lucida Console'>(</font>eps, gradient_input, means, invstds, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, g_grad, b_grad <font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> bn_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> CONV_MODE<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>bn_con2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font> <font color='#009900'>// if FC_MODE
</font>                <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>bn_fc2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gamma, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.beta, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.means, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.invstds, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.running_means, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.running_variances, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_updates, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.running_stats_window_size, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.eps, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>bn_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> CONV_MODE<font face='Lucida Console'>)</font> 
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_con2</font>"<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::bn_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#009900'>// must be in FC_MODE
</font>            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_fc2</font>"<font face='Lucida Console'>)</font>
                    <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::bn_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gamma, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.beta, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.means, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.invstds, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.running_means, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.running_variances, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_updates, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.running_stats_window_size, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.eps, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> bn_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>bn_con  </font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>bn_fc   </font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> running_stats_window_size=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.running_stats_window_size;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> bn_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;bn_con</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;bn_fc</font>";

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> running_stats_window_size='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.running_stats_window_size<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";

            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/bn_con&gt;\n</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/bn_fc&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'>friend</font> <font color='#0000FF'>class</font> <b><a name='affine_'></a>affine_</b>;

        resizable_tensor params;
        alias_tensor gamma, beta;
        resizable_tensor means, running_means;
        resizable_tensor invstds, running_variances;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_updates;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> running_stats_window_size;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> eps;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> bn_con <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>bn_<font color='#5555FF'>&lt;</font>CONV_MODE<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> bn_fc <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>bn_<font color='#5555FF'>&lt;</font>FC_MODE<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>enum</font> <b><a name='fc_bias_mode'></a>fc_bias_mode</b>
    <b>{</b>
        FC_HAS_BIAS <font color='#5555FF'>=</font> <font color='#979000'>0</font>,
        FC_NO_BIAS <font color='#5555FF'>=</font> <font color='#979000'>1</font>
    <b>}</b>;

    <font color='#0000FF'>struct</font> <b><a name='num_fc_outputs'></a>num_fc_outputs</b>
    <b>{</b>
        <b><a name='num_fc_outputs'></a>num_fc_outputs</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> n<font face='Lucida Console'>)</font> : num_outputs<font face='Lucida Console'>(</font>n<font face='Lucida Console'>)</font> <b>{</b><b>}</b>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs_,
        fc_bias_mode bias_mode
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='fc_'></a>fc_</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>num_outputs_ <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of outputs from a fc_ layer must be &gt; 0</font>"<font face='Lucida Console'>)</font>;

    <font color='#0000FF'>public</font>:
        <b><a name='fc_'></a>fc_</b><font face='Lucida Console'>(</font>num_fc_outputs o<font face='Lucida Console'>)</font> : num_outputs<font face='Lucida Console'>(</font>o.num_outputs<font face='Lucida Console'>)</font>, num_inputs<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_learning_rate_multiplier<font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>,
            bias_weight_decay_multiplier<font face='Lucida Console'>(</font><font color='#979000'>0</font><font face='Lucida Console'>)</font>,
            use_bias<font face='Lucida Console'>(</font><font color='#979000'>true</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        <b><a name='fc_'></a>fc_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> : fc_<font face='Lucida Console'>(</font>num_fc_outputs<font face='Lucida Console'>(</font>num_outputs_<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_learning_rate_multiplier'></a>get_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_weight_decay_multiplier'></a>get_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_learning_rate_multiplier'></a>set_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_weight_decay_multiplier'></a>set_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_learning_rate_multiplier'></a>get_bias_learning_rate_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>  <b>{</b> <font color='#0000FF'>return</font> bias_learning_rate_multiplier; <b>}</b>
        <font color='#0000FF'><u>double</u></font> <b><a name='get_bias_weight_decay_multiplier'></a>get_bias_weight_decay_multiplier</b> <font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>   <b>{</b> <font color='#0000FF'>return</font> bias_weight_decay_multiplier; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_learning_rate_multiplier'></a>set_bias_learning_rate_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font> <b>{</b> bias_learning_rate_multiplier <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='set_bias_weight_decay_multiplier'></a>set_bias_weight_decay_multiplier</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>double</u></font> val<font face='Lucida Console'>)</font>  <b>{</b> bias_weight_decay_multiplier  <font color='#5555FF'>=</font> val; <b>}</b>
        <font color='#0000FF'><u>void</u></font> <b><a name='disable_bias'></a>disable_bias</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> use_bias <font color='#5555FF'>=</font> <font color='#979000'>false</font>; <b>}</b>
        <font color='#0000FF'><u>bool</u></font> <b><a name='bias_is_disabled'></a>bias_is_disabled</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#5555FF'>!</font>use_bias; <b>}</b>

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> <b><a name='get_num_outputs'></a>get_num_outputs</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> num_outputs; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='set_num_outputs'></a>set_num_outputs</b><font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font> num<font face='Lucida Console'>)</font> 
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>num <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>num <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>num_outputs<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#BB00BB'>get_layer_params</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, 
                    "<font color='#CC0000'>You can't change the number of filters in fc_ if the parameter tensor has already been allocated.</font>"<font face='Lucida Console'>)</font>;
                num_outputs <font color='#5555FF'>=</font> num;
            <b>}</b>
        <b>}</b>

        fc_bias_mode <b><a name='get_bias_mode'></a>get_bias_mode</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> bias_mode; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            num_inputs <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> use_bias<font face='Lucida Console'>)</font>
                params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs<font color='#5555FF'>+</font><font color='#979000'>1</font>, num_outputs<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>num_inputs, num_outputs<font face='Lucida Console'>)</font>;

            dlib::rand <font color='#BB00BB'>rnd</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>rand</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>randomize_parameters</font><font face='Lucida Console'>(</font>params, num_inputs<font color='#5555FF'>+</font>num_outputs, rnd<font face='Lucida Console'>)</font>;

            weights <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>num_inputs, num_outputs<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                biases <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,num_outputs<font face='Lucida Console'>)</font>;
                <font color='#009900'>// set the initial bias values to zero
</font>                <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params,weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>num_inputs <font color='#5555FF'>=</font><font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                "<font color='#CC0000'>The size of the input tensor to this fc layer doesn't match the size the fc layer was trained with.</font>"<font face='Lucida Console'>)</font>;
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, num_outputs<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> w <font color='#5555FF'>=</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,output, <font color='#979000'>1</font>,sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>false</font>, w,<font color='#979000'>false</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> use_bias<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,output,<font color='#979000'>1</font>,b<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> params_grad<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// no point computing the parameter gradients if they won't be used.
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>learning_rate_multiplier <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// compute the gradient of the weight parameters.  
</font>                <font color='#0000FF'>auto</font> pw <font color='#5555FF'>=</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params_grad, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>,pw, <font color='#979000'>1</font>,sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>true</font>, gradient_input,<font color='#979000'>false</font><font face='Lucida Console'>)</font>;

                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> use_bias<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#009900'>// compute the gradient of the bias parameters.  
</font>                    <font color='#0000FF'>auto</font> pb <font color='#5555FF'>=</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params_grad, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                    tt::<font color='#BB00BB'>assign_bias_gradient</font><font face='Lucida Console'>(</font>pb, gradient_input<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>

            <font color='#009900'>// compute the gradient for the data
</font>            <font color='#0000FF'>auto</font> w <font color='#5555FF'>=</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gemm</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, <font color='#979000'>1</font>,gradient_input,<font color='#979000'>false</font>, w,<font color='#979000'>true</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_instance <b><a name='get_weights'></a>get_weights</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_const_instance <b><a name='get_weights'></a>get_weights</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#0000FF'>return</font> <font color='#BB00BB'>weights</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_instance <b><a name='get_biases'></a>get_biases</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS, "<font color='#CC0000'>This fc_ layer doesn't have a bias vector </font>"
                "<font color='#CC0000'>to be retrieved, as per template parameter 'bias_mode'.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        alias_tensor_const_instance <b><a name='get_biases'></a>get_biases</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            <font color='#BB00BB'>static_assert</font><font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS, "<font color='#CC0000'>This fc_ layer doesn't have a bias vector </font>"
                "<font color='#CC0000'>to be retrieved, as per template parameter 'bias_mode'.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>return</font> <font color='#BB00BB'>biases</font><font face='Lucida Console'>(</font>params, weights.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> fc_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>fc_3</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_outputs, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.num_inputs, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weights, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.biases, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font><font face='Lucida Console'>)</font>bias_mode, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.use_bias, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>fc_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>fc_2</font>" <font color='#5555FF'>|</font><font color='#5555FF'>|</font> version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>fc_3</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_outputs, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.num_inputs, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weights, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.biases, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'><u>int</u></font> bmode <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>bmode, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>fc_bias_mode<font face='Lucida Console'>)</font>bmode<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong fc_bias_mode found while deserializing dlib::fc_</font>"<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_learning_rate_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.bias_weight_decay_multiplier, in<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>fc_3</font>"<font face='Lucida Console'>)</font>
                <b>{</b>
                    <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.use_bias, in<font face='Lucida Console'>)</font>;
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::fc_.</font>"<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> fc_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_HAS_BIAS<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>fc\t (</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_outputs=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.use_bias<font face='Lucida Console'>)</font>
                <b>{</b>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier;
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier;
                <b>}</b>
                <font color='#0000FF'>else</font>
                <b>{</b>
                    out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> use_bias=false</font>";
                <b>}</b>
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>fc_no_bias (</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>num_outputs=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier;
            <b>}</b>
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> fc_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>bias_mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>FC_HAS_BIAS<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;fc</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_outputs='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> bias_weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.bias_weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> use_bias='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>item.use_bias?"<font color='#CC0000'>true</font>":"<font color='#CC0000'>false</font>"<font face='Lucida Console'>)</font><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'&gt;\n</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/fc&gt;\n</font>";
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;fc_no_bias</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> num_outputs='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.num_outputs<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> learning_rate_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.learning_rate_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>"
                    <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> weight_decay_mult='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.weight_decay_multiplier<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/fc_no_bias&gt;\n</font>";
            <b>}</b>
        <b>}</b>

    <font color='#0000FF'>private</font>:

        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs;
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_inputs;
        resizable_tensor params;
        alias_tensor weights, biases;
        <font color='#0000FF'><u>double</u></font> learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> weight_decay_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_learning_rate_multiplier;
        <font color='#0000FF'><u>double</u></font> bias_weight_decay_multiplier;
        <font color='#0000FF'><u>bool</u></font> use_bias;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> fc <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>fc_<font color='#5555FF'>&lt;</font>num_outputs,FC_HAS_BIAS<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> num_outputs,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> fc_no_bias <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>fc_<font color='#5555FF'>&lt;</font>num_outputs,FC_NO_BIAS<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='dropout_'></a>dropout_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='dropout_'></a>dropout_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> drop_rate_ <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>
        <font face='Lucida Console'>)</font> :
            drop_rate<font face='Lucida Console'>(</font>drop_rate_<font face='Lucida Console'>)</font>,
            rnd<font face='Lucida Console'>(</font>std::rand<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font color='#979000'>0</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> drop_rate <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> drop_rate <font color='#5555FF'>&lt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#009900'>// We have to add a copy constructor and assignment operator because the rnd object
</font>        <font color='#009900'>// is non-copyable.
</font>        <b><a name='dropout_'></a>dropout_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : drop_rate<font face='Lucida Console'>(</font>item.drop_rate<font face='Lucida Console'>)</font>, mask<font face='Lucida Console'>(</font>item.mask<font face='Lucida Console'>)</font>, rnd<font face='Lucida Console'>(</font>std::rand<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
        <b>{</b><b>}</b>

        dropout_<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#0000FF'>this</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#5555FF'>&amp;</font>item<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;

            drop_rate <font color='#5555FF'>=</font> item.drop_rate;
            mask <font color='#5555FF'>=</font> item.mask;
            <font color='#0000FF'>return</font> <font color='#5555FF'>*</font><font color='#0000FF'>this</font>;
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_drop_rate'></a>get_drop_rate</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> drop_rate; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// create a random mask and use it to filter the data
</font>            mask.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>input<font face='Lucida Console'>)</font>;
            rnd.<font color='#BB00BB'>fill_uniform</font><font face='Lucida Console'>(</font>mask<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>threshold</font><font face='Lucida Console'>(</font>mask, drop_rate<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output, input, mask<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, data_grad, mask, gradient_input<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, data_grad, mask, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>dropout_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.drop_rate, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.mask, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>dropout_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>dropout_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::dropout_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.drop_rate, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.mask, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='clean'></a>clean</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> 
        <b>{</b>
            mask.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>dropout\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>drop_rate=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.drop_rate
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;dropout</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> drop_rate='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.drop_rate<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>float</u></font> drop_rate;
        resizable_tensor mask;

        tt::tensor_rand rnd;
        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> dropout <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>dropout_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='multiply_'></a>multiply_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='multiply_'></a>multiply_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> val_ <font color='#5555FF'>=</font> <font color='#979000'>0.5</font>
        <font face='Lucida Console'>)</font> :
            val<font face='Lucida Console'>(</font>val_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <b><a name='multiply_'></a>multiply_</b> <font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> dropout_<font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font> : val<font face='Lucida Console'>(</font><font color='#979000'>1</font><font color='#5555FF'>-</font>item.get_drop_rate<font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <b>{</b><b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_multiply_value'></a>get_multiply_value</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> val; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>output, input, val<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>data_grad, gradient_input, val<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>data_grad, data_grad, gradient_input, <font color='#979000'>1</font>, val<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> multiply_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>multiply_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.val, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>multiply_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>dropout_</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Since we can build a multiply_ from a dropout_ we check if that's what
</font>                <font color='#009900'>// is in the stream and if so then just convert it right here.
</font>                unserialize <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                dropout_ temp;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>temp, sin<font face='Lucida Console'>)</font>;
                item <font color='#5555FF'>=</font> temp;
                <font color='#0000FF'>return</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>multiply_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::multiply_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.val, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> multiply_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>multiply (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>val=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.val
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> multiply_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;multiply</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> val='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.val<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>float</u></font> val;
        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> multiply <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>multiply_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='affine_'></a>affine_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='affine_'></a>affine_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> : mode<font face='Lucida Console'>(</font>FC_MODE<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <b><a name='affine_'></a>affine_</b><font face='Lucida Console'>(</font>
            layer_mode mode_
        <font face='Lucida Console'>)</font> : mode<font face='Lucida Console'>(</font>mode_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
            layer_mode bnmode
            <font color='#5555FF'>&gt;</font>
        <b><a name='affine_'></a>affine_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> bn_<font color='#5555FF'>&lt;</font>bnmode<font color='#5555FF'>&gt;</font><font color='#5555FF'>&amp;</font> item
        <font face='Lucida Console'>)</font>
        <b>{</b>
            gamma <font color='#5555FF'>=</font> item.gamma;
            beta <font color='#5555FF'>=</font> item.beta;
            mode <font color='#5555FF'>=</font> bnmode;

            params.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            resizable_tensor <font color='#BB00BB'>temp</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> sg <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>temp,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> sb <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>temp,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            g <font color='#5555FF'>=</font> <font color='#BB00BB'>pointwise_divide</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>sg<font face='Lucida Console'>)</font>, <font color='#BB00BB'>sqrt</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.running_variances<font face='Lucida Console'>)</font><font color='#5555FF'>+</font>item.<font color='#BB00BB'>get_eps</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            b <font color='#5555FF'>=</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>sb<font face='Lucida Console'>)</font> <font color='#5555FF'>-</font> <font color='#BB00BB'>pointwise_multiply</font><font face='Lucida Console'>(</font><font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>g<font face='Lucida Console'>)</font>, <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.running_means<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        layer_mode <b><a name='get_mode'></a>get_mode</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> mode; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='disable'></a>disable</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
            params.<font color='#BB00BB'>clear</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            disabled <font color='#5555FF'>=</font> <font color='#979000'>true</font>;
        <b>}</b>

        <font color='#0000FF'><u>bool</u></font> <b><a name='is_disabled'></a>is_disabled</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> disabled; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                gamma <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            beta <font color='#5555FF'>=</font> gamma;

            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>+</font>beta.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>1</font>;
            <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> <font color='#979000'>0</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
                tt::<font color='#BB00BB'>affine_transform</font><font face='Lucida Console'>(</font>output, input, g, b<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>else</font>
                tt::<font color='#BB00BB'>affine_transform_conv</font><font face='Lucida Console'>(</font>output, input, g, b<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>disabled<font face='Lucida Console'>)</font>
                <font color='#0000FF'>return</font>;

            <font color='#0000FF'>auto</font> g <font color='#5555FF'>=</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> b <font color='#5555FF'>=</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params,gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// We are computing the gradient of dot(gradient_input, computed_output*g + b)
</font>            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>mode <font color='#5555FF'>=</font><font color='#5555FF'>=</font> FC_MODE<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    tt::<font color='#BB00BB'>multiply</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                    tt::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
                <font color='#0000FF'>else</font>
                    tt::<font color='#BB00BB'>multiply_conv</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, data_grad, gradient_input, g<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        alias_tensor_instance <b><a name='get_gamma'></a>get_gamma</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>; <b>}</b>;
        alias_tensor_const_instance <b><a name='get_gamma'></a>get_gamma</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>gamma</font><font face='Lucida Console'>(</font>params, <font color='#979000'>0</font><font face='Lucida Console'>)</font>; <b>}</b>;

        alias_tensor_instance <b><a name='get_beta'></a>get_beta</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params, gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <b>}</b>;
        alias_tensor_const_instance <b><a name='get_beta'></a>get_beta</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> <font color='#BB00BB'>beta</font><font face='Lucida Console'>(</font>params, gamma.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>; <b>}</b>;

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> empty_params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> empty_params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> affine_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>affine_2</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.gamma, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.beta, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>int</u></font><font face='Lucida Console'>)</font>item.mode, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.disabled, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>affine_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_con2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Since we can build an affine_ from a bn_ we check if that's what is in
</font>                <font color='#009900'>// the stream and if so then just convert it right here.
</font>                unserialize <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                bn_<font color='#5555FF'>&lt;</font>CONV_MODE<font color='#5555FF'>&gt;</font> temp;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>temp, sin<font face='Lucida Console'>)</font>;
                item <font color='#5555FF'>=</font> temp;
                <font color='#0000FF'>return</font>;
            <b>}</b>
            <font color='#0000FF'>else</font> <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>bn_fc2</font>"<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#009900'>// Since we can build an affine_ from a bn_ we check if that's what is in
</font>                <font color='#009900'>// the stream and if so then just convert it right here.
</font>                unserialize <font color='#BB00BB'>sin</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
                bn_<font color='#5555FF'>&lt;</font>FC_MODE<font color='#5555FF'>&gt;</font> temp;
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>temp, sin<font face='Lucida Console'>)</font>;
                item <font color='#5555FF'>=</font> temp;
                <font color='#0000FF'>return</font>;
            <b>}</b>

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>affine_</font>" <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>affine_2</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::affine_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.gamma, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.beta, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>int</u></font> mode;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>mode, in<font face='Lucida Console'>)</font>;
            item.mode <font color='#5555FF'>=</font> <font face='Lucida Console'>(</font>layer_mode<font face='Lucida Console'>)</font>mode;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>=</font><font color='#5555FF'>=</font> "<font color='#CC0000'>affine_2</font>"<font face='Lucida Console'>)</font>
                <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.disabled, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> affine_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>affine</font>";
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.disabled<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>\t (disabled)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> affine_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;affine_con</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;affine_fc</font>";
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.disabled<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> disabled='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> std::boolalpha <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.disabled <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>item.mode<font color='#5555FF'>=</font><font color='#5555FF'>=</font>CONV_MODE<font face='Lucida Console'>)</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/affine_con&gt;\n</font>";
            <font color='#0000FF'>else</font>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/affine_fc&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params, empty_params; 
        alias_tensor gamma, beta;
        layer_mode mode;
        <font color='#0000FF'><u>bool</u></font> disabled <font color='#5555FF'>=</font> <font color='#979000'>false</font>;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> affine <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>affine_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='add_prev_'></a>add_prev_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>::id;

        <b><a name='add_prev_'></a>add_prev_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t1 <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t2 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>output, t1, t2<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// The gradient just flows backwards to the two layers that forward() added
</font>            <font color='#009900'>// together.
</font>            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>add</font><font face='Lucida Console'>(</font>layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>add_prev_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>add_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>add_prev_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::add_prev_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>
        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> add_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>add_prev</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> add_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;add_prev tag='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> add_prev <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>add_prev_<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev1  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag1, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev2  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag2, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev3  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag3, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev4  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag4, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev5  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag5, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev6  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag6, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev7  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag7, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev8  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag8, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev9  <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag9, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> add_prev10 <font color='#5555FF'>=</font> add_prev<font color='#5555FF'>&lt;</font>tag10, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>using</font> add_prev1_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev2_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev3_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag3<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev4_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag4<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev5_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag5<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev6_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag6<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev7_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag7<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev8_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag8<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev9_  <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag9<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> add_prev10_ <font color='#5555FF'>=</font> add_prev_<font color='#5555FF'>&lt;</font>tag10<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='mult_prev_'></a>mult_prev_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>::id;

        <b><a name='mult_prev_'></a>mult_prev_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t1 <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t2 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>,
                            std::<font color='#BB00BB'>max</font><font face='Lucida Console'>(</font>t1.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,t2.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output, t1, t2<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t1 <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> t2 <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// The gradient just flows backwards to the two layers that forward()
</font>            <font color='#009900'>// multiplied together.
</font>            tt::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, t2, gradient_input<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>multiply_zero_padded</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, t1, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> mult_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>mult_prev_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>mult_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>mult_prev_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::mult_prev_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> mult_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>mult_prev</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> mult_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;mult_prev tag='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> mult_prev <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>mult_prev_<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev1  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag1, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev2  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag2, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev3  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag3, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev4  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag4, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev5  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag5, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev6  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag6, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev7  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag7, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev8  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag8, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev9  <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag9, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> mult_prev10 <font color='#5555FF'>=</font> mult_prev<font color='#5555FF'>&lt;</font>tag10, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>using</font> mult_prev1_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev2_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev3_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag3<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev4_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag4<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev5_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag5<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev6_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag6<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev7_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag7<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev8_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag8<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev9_  <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag9<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> mult_prev10_ <font color='#5555FF'>=</font> mult_prev_<font color='#5555FF'>&lt;</font>tag10<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='resize_prev_to_tagged_'></a>resize_prev_to_tagged_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>::id;

        <b><a name='resize_prev_to_tagged_'></a>resize_prev_to_tagged_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> prev <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> tagged <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>prev.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> tagged.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>prev.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                            prev.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                            tagged.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                            tagged.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>prev.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> tagged.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> prev.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> tagged.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output, <font color='#979000'>0</font>, prev, <font color='#979000'>0</font>, prev.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                tt::<font color='#BB00BB'>resize_bilinear</font><font face='Lucida Console'>(</font>output, prev<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> prev <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;

            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>prev.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>prev.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>prev.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> prev.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> gradient_input.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, prev, <font color='#979000'>0</font>, gradient_input, <font color='#979000'>0</font>, prev.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                tt::<font color='#BB00BB'>resize_bilinear_gradient</font><font face='Lucida Console'>(</font>prev, gradient_input<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> resize_prev_to_tagged_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>resize_prev_to_tagged_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>resize_prev_to_tagged_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>resize_prev_to_tagged_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::resize_prev_to_tagged_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> resize_prev_to_tagged_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>resize_prev_to_tagged</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> resize_prev_to_tagged_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;resize_prev_to_tagged tag='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> resize_prev_to_tagged <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>resize_prev_to_tagged_<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='scale_'></a>scale_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>::id;

        <b><a name='scale_'></a>scale_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> scales <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>scales.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>           <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         scales.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>          <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         scales.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>          <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, 
                         "<font color='#CC0000'>scales.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>
                         "<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
                         <font face='Lucida Console'>)</font>;

            output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output, src, scales<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> scales <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#009900'>// The gradient just flows backwards to the two layers that forward()
</font>            <font color='#009900'>// read from.
</font>            tt::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input, scales<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>reshape_src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                reshape_scales <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                reshape_src <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> scales_grad <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> sgrad <font color='#5555FF'>=</font> <font color='#BB00BB'>reshape_scales</font><font face='Lucida Console'>(</font>scales_grad<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, sgrad, <font color='#BB00BB'>reshape_src</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>, <font color='#BB00BB'>reshape_src</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> scale_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>scale_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.reshape_scales, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.reshape_src, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>scale_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>scale_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::scale_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.reshape_scales, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.reshape_src, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> scale_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>scale</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> scale_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;scale tag='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        alias_tensor reshape_scales;
        alias_tensor reshape_src;
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> scale <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>scale_<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale1  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag1, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale2  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag2, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale3  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag3, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale4  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag4, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale5  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag5, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale6  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag6, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale7  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag7, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale8  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag8, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale9  <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag9, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale10 <font color='#5555FF'>=</font> scale<font color='#5555FF'>&lt;</font>tag10, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>using</font> scale1_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale2_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale3_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag3<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale4_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag4<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale5_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag5<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale6_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag6<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale7_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag7<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale8_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag8<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale9_  <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag9<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale10_ <font color='#5555FF'>=</font> scale_<font color='#5555FF'>&lt;</font>tag10<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='scale_prev_'></a>scale_prev_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>const</font> <font color='#0000FF'>static</font> <font color='#0000FF'><u>unsigned</u></font> <font color='#0000FF'><u>long</u></font> id <font color='#5555FF'>=</font> tag_id<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>::id;

        <b><a name='scale_prev_'></a>scale_prev_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> scales <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>scales.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>           <font color='#5555FF'>=</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         scales.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>          <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font> <font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font>
                         scales.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>          <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>,
                         "<font color='#CC0000'>scales.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> scales.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>
                         "<font color='#CC0000'>\nsrc.k(): </font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
                         <font face='Lucida Console'>)</font>;

            output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, output, src, scales<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> src <font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> scales <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>scale_channels</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input, scales<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>reshape_src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                reshape_scales <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                reshape_src <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>src.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,src.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font>src.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font><font color='#5555FF'>&amp;</font> scales_grad <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> sgrad <font color='#5555FF'>=</font> <font color='#BB00BB'>reshape_scales</font><font face='Lucida Console'>(</font>scales_grad<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, sgrad, <font color='#BB00BB'>reshape_src</font><font face='Lucida Console'>(</font>src<font face='Lucida Console'>)</font>, <font color='#BB00BB'>reshape_src</font><font face='Lucida Console'>(</font>gradient_input<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> scale_prev_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>scale_prev_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.reshape_scales, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.reshape_src, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>scale_prev_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>scale_prev_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::scale_prev_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.reshape_scales, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.reshape_src, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> scale_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>scale_prev</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> scale_prev_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;scale_prev tag='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>id<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        alias_tensor reshape_scales;
        alias_tensor reshape_src;
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='tag'></a>tag</b>,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> scale_prev <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>scale_prev_<font color='#5555FF'>&lt;</font>tag<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev1  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag1, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev2  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag2, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev3  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag3, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev4  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag4, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev5  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag5, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev6  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag6, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev7  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag7, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev8  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag8, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev9  <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag9, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> scale_prev10 <font color='#5555FF'>=</font> scale_prev<font color='#5555FF'>&lt;</font>tag10, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>using</font> scale_prev1_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag1<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev2_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag2<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev3_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag3<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev4_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag4<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev5_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag5<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev6_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag6<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev7_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag7<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev8_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag8<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev9_  <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag9<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>using</font> scale_prev10_ <font color='#5555FF'>=</font> scale_prev_<font color='#5555FF'>&lt;</font>tag10<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='relu_'></a>relu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='relu_'></a>relu_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>relu</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>relu_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>relu_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>relu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>relu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::relu_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>relu</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> relu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;relu/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> relu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>relu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='prelu_'></a>prelu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='prelu_'></a>prelu_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> initial_param_value_ <font color='#5555FF'>=</font> <font color='#979000'>0.25</font>
        <font face='Lucida Console'>)</font> : initial_param_value<font face='Lucida Console'>(</font>initial_param_value_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_initial_param_value'></a>get_initial_param_value</b> <font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> initial_param_value; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            params.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font><font color='#979000'>1</font><font face='Lucida Console'>)</font>;
            params <font color='#5555FF'>=</font> initial_param_value;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, 
            resizable_tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font>
        <b>{</b>
            data_output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>prelu</font><font face='Lucida Console'>(</font>data_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, params<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            SUBNET<font color='#5555FF'>&amp;</font> sub, 
            tensor<font color='#5555FF'>&amp;</font> params_grad
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>prelu_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, 
                gradient_input, params, params_grad<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> prelu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>prelu_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.params, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.initial_param_value, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>prelu_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>prelu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::prelu_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.params, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.initial_param_value, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> prelu_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>prelu\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>initial_param_value=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.initial_param_value
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> prelu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;prelu initial_param_value='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.initial_param_value<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font color='#BB00BB'>mat</font><font face='Lucida Console'>(</font>item.params<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;/prelu&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>float</u></font> initial_param_value;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> prelu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>prelu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='leaky_relu_'></a>leaky_relu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='leaky_relu_'></a>leaky_relu_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> alpha_ <font color='#5555FF'>=</font> <font color='#979000'>0.01f</font>
        <font face='Lucida Console'>)</font> : alpha<font face='Lucida Console'>(</font>alpha_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_alpha'></a>get_alpha</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b>
            <font color='#0000FF'>return</font> alpha;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>leaky_relu</font><font face='Lucida Console'>(</font>output, input, alpha<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            tensor<font color='#5555FF'>&amp;</font> data_grad,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>leaky_relu_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input, alpha<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> leaky_relu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>leaky_relu_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.alpha, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>leaky_relu_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>leaky_relu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::leaky_relu_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.alpha, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> leaky_relu_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>leaky_relu\t(</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>alpha=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.alpha
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> leaky_relu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;leaky_relu alpha='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.alpha <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;leaky_relu/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>float</u></font> alpha;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> leaky_relu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>leaky_relu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='sig_'></a>sig_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='sig_'></a>sig_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>sigmoid</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>sigmoid_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> sig_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>sig_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>sig_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>sig_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::sig_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> sig_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>sig</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> sig_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;sig/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> sig <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>sig_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='mish_'></a>mish_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='mish_'></a>mish_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub,
            resizable_tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font>
        <b>{</b>
            data_output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>mish</font><font face='Lucida Console'>(</font>data_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            SUBNET<font color='#5555FF'>&amp;</font> sub,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>mish_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> mish_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>mish_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>mish_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>mish_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::mish_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> mish_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>mish</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> mish_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;mish/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> mish <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>mish_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='htan_'></a>htan_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='htan_'></a>htan_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>tanh</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>tanh_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> htan_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>htan_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>htan_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>htan_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::htan_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> htan_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>htan</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> htan_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;htan/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;


    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> htan <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>htan_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='clipped_relu_'></a>clipped_relu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='clipped_relu_'></a>clipped_relu_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> ceiling_ <font color='#5555FF'>=</font> <font color='#979000'>6.0f</font>
        <font face='Lucida Console'>)</font> : ceiling<font face='Lucida Console'>(</font>ceiling_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_ceiling'></a>get_ceiling</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b>
            <font color='#0000FF'>return</font> ceiling;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>clipped_relu</font><font face='Lucida Console'>(</font>output, input, ceiling<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            tensor<font color='#5555FF'>&amp;</font> data_grad,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>clipped_relu_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input, ceiling<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> clipped_relu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>clipped_relu_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.ceiling, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>clipped_relu_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>clipped_relu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::clipped_relu_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.ceiling, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> clipped_relu_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>clipped_relu\t(</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>ceiling=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.ceiling
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> clipped_relu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;clipped_relu ceiling='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.ceiling <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>float</u></font> ceiling;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> clipped_relu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>clipped_relu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='elu_'></a>elu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='elu_'></a>elu_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> <font color='#0000FF'><u>float</u></font> alpha_ <font color='#5555FF'>=</font> <font color='#979000'>1.0f</font>
        <font face='Lucida Console'>)</font> : alpha<font face='Lucida Console'>(</font>alpha_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_alpha'></a>get_alpha</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b>
            <font color='#0000FF'>return</font> alpha;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>elu</font><font face='Lucida Console'>(</font>output, input, alpha<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            tensor<font color='#5555FF'>&amp;</font> data_grad,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>elu_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input, alpha<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> elu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>elu_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.alpha, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>elu_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>elu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::elu_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.alpha, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> elu_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>elu\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>alpha=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.alpha
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> elu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;elu alpha='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.alpha <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>float</u></font> alpha;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> elu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>elu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='gelu_'></a>gelu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='gelu_'></a>gelu_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub,
            resizable_tensor<font color='#5555FF'>&amp;</font> data_output
        <font face='Lucida Console'>)</font>
        <b>{</b>
            data_output.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>gelu</font><font face='Lucida Console'>(</font>data_output, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            SUBNET<font color='#5555FF'>&amp;</font> sub,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>gelu_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> gelu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>gelu_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>gelu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>gelu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::gelu_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> gelu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>gelu</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> gelu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;gelu/&gt;\n</font>";
        <b>}</b>


    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> gelu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>gelu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='smelu_'></a>smelu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='smelu_'></a>smelu_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>float</u></font> beta_ <font color='#5555FF'>=</font> <font color='#979000'>1</font>
        <font face='Lucida Console'>)</font> : beta<font face='Lucida Console'>(</font>beta_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>float</u></font> <b><a name='get_beta'></a>get_beta</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b>
            <font color='#0000FF'>return</font> beta;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>smelu</font><font face='Lucida Console'>(</font>output, input, beta<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            tensor<font color='#5555FF'>&amp;</font> data_grad,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>smelu_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input, beta<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> smelu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>smelu_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.beta, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>smelu_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>smelu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::smelu_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.beta, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> smelu_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>smelu\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>beta=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.beta
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> smelu_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;smelu beta='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> item.beta <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'&gt;\n</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;smelu/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
        <font color='#0000FF'><u>float</u></font> beta;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> smelu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>smelu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='silu_'></a>silu_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='silu_'></a>silu_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub,
            resizable_tensor<font color='#5555FF'>&amp;</font> data_ouput<font face='Lucida Console'>)</font>
        <b>{</b>
            data_ouput.<font color='#BB00BB'>copy_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>silu</font><font face='Lucida Console'>(</font>data_ouput, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input,
            SUBNET<font color='#5555FF'>&amp;</font> sub,
            tensor<font color='#5555FF'>&amp;</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>silu_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> dpoint<font color='#5555FF'>&amp;</font> p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> silu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>silu_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>silu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>silu_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::silu_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> silu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>silu</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> silu_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;silu/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> silu <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>silu_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='softmax_'></a>softmax_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='softmax_'></a>softmax_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>softmax</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>softmax_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> softmax_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>softmax_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>softmax_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>softmax_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::softmax_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> softmax_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>softmax</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> softmax_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;softmax/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> softmax <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>softmax_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>class</font> <b><a name='softmax_all_'></a>softmax_all_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <b><a name='softmax_all_'></a>softmax_all_</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> 
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>softmax_all</font><font face='Lucida Console'>(</font>output, input<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output,
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> 
        <font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>softmax_all_gradient</font><font face='Lucida Console'>(</font>data_grad, computed_output, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> softmax_all_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>softmax_all_</font>", out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>softmax_all_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>softmax_all_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::softmax_all_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> softmax_all_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>softmax_all</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> softmax_all_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;softmax_all/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> softmax_all <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>softmax_all_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>namespace</font> impl
    <b>{</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b>, <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font>... <b><a name='TAG_TYPES'></a>TAG_TYPES</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='concat_helper_impl'></a>concat_helper_impl</b><b>{</b>

            constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><font color='#0000FF'>return</font> <font color='#979000'>1</font> <font color='#5555FF'>+</font> concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;<b>}</b>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
            <b>{</b>
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font> <font color='#979000'>1</font> ? "<font color='#CC0000'>,</font>" : "<font color='#CC0000'></font>"<font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            <b>}</b>

            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='resize_out'></a>resize_out</b><font face='Lucida Console'>(</font>resizable_tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>long</u></font> sum_k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>resize_out</font><font face='Lucida Console'>(</font>out, sub, sum_k <font color='#5555FF'>+</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='concat'></a>concat</b><font face='Lucida Console'>(</font>tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, out, k_offset, t, <font color='#979000'>0</font>, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                k_offset <font color='#5555FF'>+</font><font color='#5555FF'>=</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>concat</font><font face='Lucida Console'>(</font>out, sub, k_offset<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='split'></a>split</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, t, <font color='#979000'>0</font>, input, k_offset, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
                k_offset <font color='#5555FF'>+</font><font color='#5555FF'>=</font> t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>input, sub, k_offset<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG_TYPE'></a>TAG_TYPE</b><font color='#5555FF'>&gt;</font>
        <font color='#0000FF'>struct</font> <b><a name='concat_helper_impl'></a>concat_helper_impl</b><font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><b>{</b>
            constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><font color='#0000FF'>return</font> <font color='#979000'>1</font>;<b>}</b>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font> 
            <b>{</b> 
                out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> tag_id<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font>::id;
            <b>}</b>

            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='resize_out'></a>resize_out</b><font face='Lucida Console'>(</font>resizable_tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>long</u></font> sum_k<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                out.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>t.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font> sum_k, t.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, t.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='concat'></a>concat</b><font face='Lucida Console'>(</font>tensor<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, out, k_offset, t, <font color='#979000'>0</font>, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
            <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='split'></a>split</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, SUBNET<font color='#5555FF'>&amp;</font> sub, <font color='#0000FF'><u>size_t</u></font> k_offset<font face='Lucida Console'>)</font>
            <b>{</b>
                <font color='#0000FF'>auto</font><font color='#5555FF'>&amp;</font> t <font color='#5555FF'>=</font> layer<font color='#5555FF'>&lt;</font>TAG_TYPE<font color='#5555FF'>&gt;</font><font face='Lucida Console'>(</font>sub<font face='Lucida Console'>)</font>.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, t, <font color='#979000'>0</font>, input, k_offset, t.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>;
    <b>}</b>
    <font color='#009900'>// concat layer
</font>    <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font>
        <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font>... <b><a name='TAG_TYPES'></a>TAG_TYPES</b>
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='concat_'></a>concat_</b>
    <b>{</b>
        <font color='#0000FF'>static</font> <font color='#0000FF'><u>void</u></font> <b><a name='list_tags'></a>list_tags</b><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font> <b>{</b> impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;<b>}</b>;

    <font color='#0000FF'>public</font>:
        constexpr <font color='#0000FF'>static</font> <font color='#0000FF'><u>size_t</u></font> <b><a name='tag_count'></a>tag_count</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b><font color='#0000FF'>return</font> impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;<b>}</b>;

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// do nothing
</font>        <b>}</b>
        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// the total depth of result is the sum of depths from all tags
</font>            impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>resize_out</font><font face='Lucida Console'>(</font>output, sub, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;

            <font color='#009900'>// copy output from each tag into different part result
</font>            impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>concat</font><font face='Lucida Console'>(</font>output, sub, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#009900'>// Gradient is split into parts for each tag layer
</font>            impl::concat_helper_impl<font color='#5555FF'>&lt;</font>TAG_TYPES...<font color='#5555FF'>&gt;</font>::<font color='#BB00BB'>split</font><font face='Lucida Console'>(</font>gradient_input, sub, <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b><font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>
        dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b><font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> p; <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> concat_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>concat_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count <font color='#5555FF'>=</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>count, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>concat_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>concat_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::concat_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>size_t</u></font> count_tags;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>count_tags, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>count_tags <font color='#5555FF'>!</font><font color='#5555FF'>=</font> <font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Invalid count of tags </font>"<font color='#5555FF'>+</font> std::<font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font>count_tags<font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>"<font color='#CC0000'>, expecting </font>" <font color='#5555FF'>+</font>
                                          std::<font color='#BB00BB'>to_string</font><font face='Lucida Console'>(</font><font color='#BB00BB'>tag_count</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font> <font color='#5555FF'>+</font>
                                                  "<font color='#CC0000'> found while deserializing dlib::concat_.</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> concat_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>concat\t (</font>";
            <font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> concat_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;concat tags='</font>";
            <font color='#BB00BB'>list_tags</font><font face='Lucida Console'>(</font>out<font face='Lucida Console'>)</font>;
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;


    <font color='#009900'>// concat layer definitions
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat2 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG3'></a>TAG3</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat3 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2, TAG3<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG3'></a>TAG3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG4'></a>TAG4</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat4 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2, TAG3, TAG4<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG1'></a>TAG1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG2'></a>TAG2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG3'></a>TAG3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG4'></a>TAG4</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font> <font color='#0000FF'>class</font> <b><a name='TAG5'></a>TAG5</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> concat5 <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>concat_<font color='#5555FF'>&lt;</font>TAG1, TAG2, TAG3, TAG4, TAG5<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#009900'>// inception layer will use tags internally. If user will use tags too, some conflicts
</font>    <font color='#009900'>// possible to exclude them, here are new tags specially for inceptions
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag0  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>0</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag1  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>1</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag2  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>2</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag3  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>3</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag4  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>4</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> itag5  <font color='#5555FF'>=</font> add_tag_layer<font color='#5555FF'>&lt;</font> <font color='#979000'>1000</font> <font color='#5555FF'>+</font> <font color='#979000'>5</font>, SUBNET<font color='#5555FF'>&gt;</font>;
    <font color='#009900'>// skip to inception input
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font> <font color='#0000FF'>using</font> iskip  <font color='#5555FF'>=</font> add_skip_layer<font color='#5555FF'>&lt;</font> itag0, SUBNET<font color='#5555FF'>&gt;</font>;

    <font color='#009900'>// here are some templates to be used for creating inception layer groups
</font>    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception2 <font color='#5555FF'>=</font> concat2<font color='#5555FF'>&lt;</font>itag1, itag2, itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font> itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B3'></a>B3</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception3 <font color='#5555FF'>=</font> concat3<font color='#5555FF'>&lt;</font>itag1, itag2, itag3, itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag3<font color='#5555FF'>&lt;</font>B3<font color='#5555FF'>&lt;</font>  itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B3'></a>B3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B4'></a>B4</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception4 <font color='#5555FF'>=</font> concat4<font color='#5555FF'>&lt;</font>itag1, itag2, itag3, itag4,
                itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag3<font color='#5555FF'>&lt;</font>B3<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font>  itag4<font color='#5555FF'>&lt;</font>B4<font color='#5555FF'>&lt;</font>  itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B1'></a>B1</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B2'></a>B2</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B3'></a>B3</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B4'></a>B4</b>,
            <font color='#0000FF'>template</font><font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font><font color='#5555FF'>&gt;</font><font color='#0000FF'>class</font> <b><a name='B5'></a>B5</b>,
            <font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> inception5 <font color='#5555FF'>=</font> concat5<font color='#5555FF'>&lt;</font>itag1, itag2, itag3, itag4, itag5,
                itag1<font color='#5555FF'>&lt;</font>B1<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag2<font color='#5555FF'>&lt;</font>B2<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font> itag3<font color='#5555FF'>&lt;</font>B3<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font>  itag4<font color='#5555FF'>&lt;</font>B4<font color='#5555FF'>&lt;</font>iskip<font color='#5555FF'>&lt;</font>  itag5<font color='#5555FF'>&lt;</font>B5<font color='#5555FF'>&lt;</font>  itag0<font color='#5555FF'>&lt;</font>SUBNET<font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font><font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font><font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>const</font> <font color='#0000FF'><u>double</u></font> DEFAULT_L2_NORM_EPS <font color='#5555FF'>=</font> <font color='#979000'>1e</font><font color='#5555FF'>-</font><font color='#979000'>5</font>;

    <font color='#0000FF'>class</font> <b><a name='l2normalize_'></a>l2normalize_</b>
    <b>{</b>
    <font color='#0000FF'>public</font>:
        <font color='#0000FF'>explicit</font> <b><a name='l2normalize_'></a>l2normalize_</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'><u>double</u></font> eps_ <font color='#5555FF'>=</font> DEFAULT_L2_NORM_EPS
        <font face='Lucida Console'>)</font> : 
            eps<font face='Lucida Console'>(</font>eps_<font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>double</u></font> <b><a name='get_eps'></a>get_eps</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> eps; <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> <font color='#009900'>/*sub*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'><u>void</u></font> <b><a name='forward_inplace'></a>forward_inplace</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> input, tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>inverse_norms</font><font face='Lucida Console'>(</font>norm, input, eps<font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>scale_rows</font><font face='Lucida Console'>(</font>output, input, norm<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'><u>void</u></font> <b><a name='backward_inplace'></a>backward_inplace</b><font face='Lucida Console'>(</font>
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> computed_output, 
            <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, 
            tensor<font color='#5555FF'>&amp;</font> data_grad, 
            tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font><font color='#BB00BB'>is_same_object</font><font face='Lucida Console'>(</font>gradient_input, data_grad<font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                tt::<font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font>temp, gradient_input, computed_output<font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>scale_rows2</font><font face='Lucida Console'>(</font><font color='#979000'>0</font>, data_grad, gradient_input, computed_output, temp, norm<font face='Lucida Console'>)</font>;
            <b>}</b>
            <font color='#0000FF'>else</font>
            <b>{</b>
                tt::<font color='#BB00BB'>dot_prods</font><font face='Lucida Console'>(</font>temp, gradient_input, computed_output<font face='Lucida Console'>)</font>;
                tt::<font color='#BB00BB'>scale_rows2</font><font face='Lucida Console'>(</font><font color='#979000'>1</font>, data_grad, gradient_input, computed_output, temp, norm<font face='Lucida Console'>)</font>;
            <b>}</b>
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> l2normalize_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>l2normalize_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>item.eps, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>l2normalize_<font color='#5555FF'>&amp;</font> item, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>l2normalize_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::l2normalize_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>item.eps, in<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> l2normalize_<font color='#5555FF'>&amp;</font> item<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>l2normalize</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps;
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> l2normalize_<font color='#5555FF'>&amp;</font> item, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;l2normalize</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> eps='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>item.eps<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:
        <font color='#0000FF'><u>double</u></font> eps;

        resizable_tensor params; <font color='#009900'>// unused
</font>        <font color='#009900'>// Here only to avoid reallocation and as a cache between forward/backward
</font>        <font color='#009900'>// functions.  
</font>        resizable_tensor norm;
        resizable_tensor temp;
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> l2normalize <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>l2normalize_, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> _offset,
        <font color='#0000FF'><u>long</u></font> _k,
        <font color='#0000FF'><u>long</u></font> _nr,
        <font color='#0000FF'><u>long</u></font> _nc
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='extract_'></a>extract_</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_offset <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The offset must be &gt;= 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_k <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>,  "<font color='#CC0000'>The number of channels must be &gt; 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nr <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of rows must be &gt; 0.</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>_nc <font color='#5555FF'>&gt;</font> <font color='#979000'>0</font>, "<font color='#CC0000'>The number of columns must be &gt; 0.</font>"<font face='Lucida Console'>)</font>;
    <font color='#0000FF'>public</font>:
        <b><a name='extract_'></a>extract_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>  
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font><font face='Lucida Console'>(</font><font color='#0000FF'><u>long</u></font><font face='Lucida Console'>)</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>*</font><font face='Lucida Console'>(</font>_offset<font color='#5555FF'>+</font>_k<font color='#5555FF'>*</font>_nr<font color='#5555FF'>*</font>_nc<font face='Lucida Console'>)</font>, 
                "<font color='#CC0000'>The tensor we are trying to extract from the input tensor is too big to fit into the input tensor.</font>"<font face='Lucida Console'>)</font>;

            aout <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, _k<font color='#5555FF'>*</font>_nr<font color='#5555FF'>*</font>_nc<font face='Lucida Console'>)</font>;
            ain <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>aout.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>!</font><font color='#5555FF'>=</font> sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>
            <b>{</b>
                aout <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, _k<font color='#5555FF'>*</font>_nr<font color='#5555FF'>*</font>_nc<font face='Lucida Console'>)</font>;
                ain <font color='#5555FF'>=</font> <font color='#BB00BB'>alias_tensor</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,  sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>size</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font color='#5555FF'>/</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
            <b>}</b>

            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, _k, _nr, _nc<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> out <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>output,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> <font color='#BB00BB'>ain</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>false</font>, out, <font color='#979000'>0</font>, in, _offset, _k<font color='#5555FF'>*</font>_nr<font color='#5555FF'>*</font>_nc<font face='Lucida Console'>)</font>;
        <b>}</b> 

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#0000FF'>auto</font> out <font color='#5555FF'>=</font> <font color='#BB00BB'>ain</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#0000FF'>auto</font> in <font color='#5555FF'>=</font> <font color='#BB00BB'>aout</font><font face='Lucida Console'>(</font>gradient_input,<font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>copy_tensor</font><font face='Lucida Console'>(</font><font color='#979000'>true</font>, out, _offset, in, <font color='#979000'>0</font>, _k<font color='#5555FF'>*</font>_nr<font color='#5555FF'>*</font>_nc<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> extract_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>extract_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_offset, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_k, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nr, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>_nc, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>extract_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>extract_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::extract_.</font>"<font face='Lucida Console'>)</font>;

            <font color='#0000FF'><u>long</u></font> offset;
            <font color='#0000FF'><u>long</u></font> k;
            <font color='#0000FF'><u>long</u></font> nr;
            <font color='#0000FF'><u>long</u></font> nc;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>offset, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>k, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nr, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>nc, in<font face='Lucida Console'>)</font>;

            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>offset <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _offset<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong offset found while deserializing dlib::extract_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>k <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _k<font face='Lucida Console'>)</font>   <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong k found while deserializing dlib::extract_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nr <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nr<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nr found while deserializing dlib::extract_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>nc <font color='#5555FF'>!</font><font color='#5555FF'>=</font> _nc<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong nc found while deserializing dlib::extract_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> extract_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>extract\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>offset=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_offset
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, k=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_k
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nr=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, nc=</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> extract_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;extract</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> offset='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_offset<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> k='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_k<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nr='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nr<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> nc='</font>"<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>_nc<font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font>"<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>
    <font color='#0000FF'>private</font>:
        alias_tensor aout, ain;

        resizable_tensor params; <font color='#009900'>// unused
</font>    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font>
        <font color='#0000FF'><u>long</u></font> offset,
        <font color='#0000FF'><u>long</u></font> k,
        <font color='#0000FF'><u>long</u></font> nr,
        <font color='#0000FF'><u>long</u></font> nc,
        <font color='#0000FF'>typename</font> SUBNET
        <font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> extract <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>extract_<font color='#5555FF'>&lt;</font>offset,k,nr,nc<font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> row_stride <font color='#5555FF'>=</font> <font color='#979000'>2</font>, <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> col_stride <font color='#5555FF'>=</font> <font color='#979000'>2</font><font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>class</font> <b><a name='reorg_'></a>reorg_</b>
    <b>{</b>
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>row_stride <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>The row_stride must be &gt;= 1</font>"<font face='Lucida Console'>)</font>;
        <b><a name='static_assert'></a>static_assert</b><font face='Lucida Console'>(</font>row_stride <font color='#5555FF'>&gt;</font><font color='#5555FF'>=</font> <font color='#979000'>1</font>, "<font color='#CC0000'>The col_stride must be &gt;= 1</font>"<font face='Lucida Console'>)</font>;

    <font color='#0000FF'>public</font>:
        <b><a name='reorg_'></a>reorg_</b><font face='Lucida Console'>(</font>
        <font face='Lucida Console'>)</font>
        <b>{</b>
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='setup'></a>setup</b> <font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> row_stride <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>DLIB_CASSERT</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>%</font> col_stride <font color='#5555FF'>=</font><font color='#5555FF'>=</font> <font color='#979000'>0</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='forward'></a>forward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> SUBNET<font color='#5555FF'>&amp;</font> sub, resizable_tensor<font color='#5555FF'>&amp;</font> output<font face='Lucida Console'>)</font>
        <b>{</b>
            output.<font color='#BB00BB'>set_size</font><font face='Lucida Console'>(</font>
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>num_samples</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>,
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>k</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> col_stride <font color='#5555FF'>*</font> row_stride,
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nr</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> row_stride,
                sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>.<font color='#BB00BB'>nc</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> col_stride
            <font face='Lucida Console'>)</font>;
            tt::<font color='#BB00BB'>reorg</font><font face='Lucida Console'>(</font>output, row_stride, col_stride, sub.<font color='#BB00BB'>get_output</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font><font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
        <font color='#0000FF'><u>void</u></font> <b><a name='backward'></a>backward</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> gradient_input, SUBNET<font color='#5555FF'>&amp;</font> sub, tensor<font color='#5555FF'>&amp;</font> <font color='#009900'>/*params_grad*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            tt::<font color='#BB00BB'>reorg_gradient</font><font face='Lucida Console'>(</font>sub.<font color='#BB00BB'>get_gradient_input</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font>, row_stride, col_stride, gradient_input<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>inline</font> dpoint <b><a name='map_input_to_output'></a>map_input_to_output</b> <font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> col_stride;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>/</font> row_stride;
            <font color='#0000FF'>return</font> p;
        <b>}</b>
        <font color='#0000FF'>inline</font> dpoint <b><a name='map_output_to_input'></a>map_output_to_input</b> <font face='Lucida Console'>(</font>dpoint p<font face='Lucida Console'>)</font> <font color='#0000FF'>const</font>
        <b>{</b>
            p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>x</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> col_stride;
            p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>=</font> p.<font color='#BB00BB'>y</font><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#5555FF'>*</font> row_stride;
            <font color='#0000FF'>return</font> p;
        <b>}</b>

        <font color='#0000FF'>const</font> tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <font color='#0000FF'>const</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>
        tensor<font color='#5555FF'>&amp;</font> <b><a name='get_layer_params'></a>get_layer_params</b><font face='Lucida Console'>(</font><font face='Lucida Console'>)</font> <b>{</b> <font color='#0000FF'>return</font> params; <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='serialize'></a>serialize</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> reorg_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>reorg_</font>", out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>row_stride, out<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>serialize</font><font face='Lucida Console'>(</font>col_stride, out<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='deserialize'></a>deserialize</b><font face='Lucida Console'>(</font>reorg_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font>, std::istream<font color='#5555FF'>&amp;</font> in<font face='Lucida Console'>)</font>
        <b>{</b>
            std::string version;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>version, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>version <font color='#5555FF'>!</font><font color='#5555FF'>=</font> "<font color='#CC0000'>reorg_</font>"<font face='Lucida Console'>)</font>
                <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Unexpected version '</font>"<font color='#5555FF'>+</font>version<font color='#5555FF'>+</font>"<font color='#CC0000'>' found while deserializing dlib::reorg_.</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> rs;
            <font color='#0000FF'><u>long</u></font> <font color='#0000FF'><u>long</u></font> cs;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>rs, in<font face='Lucida Console'>)</font>;
            <font color='#BB00BB'>deserialize</font><font face='Lucida Console'>(</font>cs, in<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>rs <font color='#5555FF'>!</font><font color='#5555FF'>=</font> row_stride<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong row_stride found while deserializing dlib::reorg_</font>"<font face='Lucida Console'>)</font>;
            <font color='#0000FF'>if</font> <font face='Lucida Console'>(</font>cs <font color='#5555FF'>!</font><font color='#5555FF'>=</font> col_stride<font face='Lucida Console'>)</font> <font color='#0000FF'>throw</font> <font color='#BB00BB'>serialization_error</font><font face='Lucida Console'>(</font>"<font color='#CC0000'>Wrong col_stride found while deserializing dlib::reorg_</font>"<font face='Lucida Console'>)</font>;
        <b>}</b>

        <font color='#0000FF'>friend</font> std::ostream<font color='#5555FF'>&amp;</font> <b><a name='operator'></a>operator</b><font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font><font face='Lucida Console'>(</font>std::ostream<font color='#5555FF'>&amp;</font> out, <font color='#0000FF'>const</font> reorg_<font color='#5555FF'>&amp;</font> <font color='#009900'>/*item*/</font><font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>reorg\t (</font>"
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>row_stride=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> row_stride
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>, col_stride=</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> col_stride
                <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>)</font>";
            <font color='#0000FF'>return</font> out;
        <b>}</b>

        <font color='#0000FF'>friend</font> <font color='#0000FF'><u>void</u></font> <b><a name='to_xml'></a>to_xml</b><font face='Lucida Console'>(</font><font color='#0000FF'>const</font> reorg_ <font color='#009900'>/*item*/</font>, std::ostream<font color='#5555FF'>&amp;</font> out<font face='Lucida Console'>)</font>
        <b>{</b>
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>&lt;reorg</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> row_stride='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> row_stride <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'> col_stride='</font>" <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> col_stride <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>'</font>";
            out <font color='#5555FF'>&lt;</font><font color='#5555FF'>&lt;</font> "<font color='#CC0000'>/&gt;\n</font>";
        <b>}</b>

    <font color='#0000FF'>private</font>:
        resizable_tensor params; <font color='#009900'>// unused
</font>
    <b>}</b>;

    <font color='#0000FF'>template</font> <font color='#5555FF'>&lt;</font><font color='#0000FF'>typename</font> SUBNET<font color='#5555FF'>&gt;</font>
    <font color='#0000FF'>using</font> reorg <font color='#5555FF'>=</font> add_layer<font color='#5555FF'>&lt;</font>reorg_<font color='#5555FF'>&lt;</font><font color='#979000'>2</font>, <font color='#979000'>2</font><font color='#5555FF'>&gt;</font>, SUBNET<font color='#5555FF'>&gt;</font>;

<font color='#009900'>// ----------------------------------------------------------------------------------------
</font>
<b>}</b>

<font color='#0000FF'>#endif</font> <font color='#009900'>// DLIB_DNn_LAYERS_H_
</font>


</pre></body></html>